# -*- coding: utf-8 -*-
import scrapy
import json,time
import pymysql
from Appstore.items import AppstoreItem
'''parse方法解析返回的数据(response data)，提取数据(生成item)以及生成需要进一步处理的URL的 Request 对象。'''
class AppstoreSpider(scrapy.Spider):
    name = 'appstore'
    allowed_domains = ['sj.qq.com']
    #start_urls = ['http://sj.qq.com/']
    baseUrl="https://sj.qq.com/myapp/cate/appList.htm?pageSize=20&orgame="
    detailUrl="https://sj.qq.com/myapp/detail.htm?apkName="
    offset=0
    orgame=1   #应用类型 1表示工具 2表示游戏
    category=-10   #应用宝的第一个categoryId
    apkName='com.tencent.mm'     #应用的包名
    start_urls=[baseUrl+str(orgame)+'&pageContext='+str(offset)+'&categoryId='+str(category),detailUrl+str(apkName)]
    # start_urls=[detailUrl+str(apkName)]
    #工具类应用分类  不适合分类：购物、阅读、新闻、社交、理财、娱乐
    lis1=[-10,103,108,115,101,119,104,117,107,112,118,111,109,100,113,116]
    # lis1=[103]
    # 记录当前长度
    l1=1

    # # 爬取流程
    def parse(self,response):
        data_list=(json.loads(response.body))['obj']
        if (json.loads(response.body)['count']==0) and (self.orgame==1) and self.l1 >=len(self.lis1):
            return
        elif (json.loads(response.body)['count']==0) and (self.orgame==1) and self.l1 <len(self.lis1):
            self.category=self.lis1[self.l1]
            self.offset=0
            self.l1=self.l1+1
        else:
            if data_list is None:
                pass
            else:
                for data in data_list:
                    item=AppstoreItem()
                    item['appName']=data['appName']
                    item['authorName']=data['authorName']
                    item['categoryName']=data['categoryName']
                    item['pkgName']=data['pkgName']
                    item['apkUrl']=data['apkUrl']
                    item['iconUrl']=data['iconUrl']
                    # 四舍五入保留一位小数
                    average_rating=data['appRatingInfo']['averageRating']
                    average_rating = '%.1f' % average_rating
                    item['averageRating']=average_rating
                    item['appDownCount']=data['appDownCount']
                    item['images']=data['images']
                    # 把时间戳转为日期格式
                    time_stamp=data['apkPublishTime']
                    time_array=time.localtime(time_stamp)
                    apk_publishtime=time.strftime('%Y-%m-%d',time_array)
                    item['apkPublishTime']=apk_publishtime
                    item['versionName']=data['versionName']
                      # 赋值apkName
                    self.apkName=data['pkgName']
                    yield scrapy.Request(self.detailUrl+self.apkName,meta={"item":item},callback=self.parseA,dont_filter=True)
                    # yield item
            self.offset=self.offset+20
        yield scrapy.Request(self.baseUrl+str(self.orgame)+'&pageContext='+str(self.offset)+'&categoryId='+str(self.category),callback=self.parse)
        

    def parseA(self,response):
        item=response.meta["item"]
        # item=AppstoreItem()
        # item['advBox']=response.xpath('//div[@id="J_AdvBox"]/text()').extract()
        appDetail=response.xpath('normalize-space(//div[@class="det-app-data-info"]/text())').extract()
        appDetail_new=appDetail[0].replace("'",'')
        item['appDetail']="'"+appDetail_new+"'"

        item['permissionInfo']=response.xpath('//ul[@class="det-othinfo-plist"]//li[@class="clearfix"]/div[2]/text()').extract()
        
        # 数据过滤处理  过滤不适合分类
        if item['categoryName']=='视频' or item['categoryName']=='旅游' or item['categoryName']=='工具' or item['categoryName']=='音乐' or item['categoryName']=='美化' or item['categoryName']=='摄影'or item['categoryName']=='系统'or item['categoryName']=='生活'or item['categoryName']=='出行'or item['categoryName']=='安全'or item['categoryName']=='教育'or item['categoryName']=='健康'or item['categoryName']=='儿童'or item['categoryName']=='办公'or item['categoryName']=='通讯':
            pass
        else:
            del item
        
        yield item
        # self.into_mysql(self,item)



        